import hashlib
import os

filedir = 'F:\\BaiduNetdiskWorkspace\\05视频图片'

files = []


def file_count(DIR):
    for name in os.listdir(DIR):
        if os.path.isfile(os.path.join(DIR, name)):
            files.append(os.path.join(DIR, name))
        if os.path.isdir(os.path.join(DIR, name)):
            file_count(os.path.join(DIR, name))
    return files


def md5_file(file):
    with open(file, 'rb') as fp:
        md5 = hashlib.md5()
        while True:
            fb = fp.read(8096)
            if not fb:
                break
            md5.update(fb)
    return md5.hexdigest()


def delfile():
    all_md5 = []
    dir = os.walk(filedir)
    j=0
    for i in dir:
        for file_name in i[2]:
            j += 1
            file = os.path.join(i[0], file_name)
            if md5_file(file) in all_md5:
                try:
                    os.remove(file)
                    print("\n删除:",file,flush=True)
                except Exception as e:
                    pass
            else:
                print("\rall_md5列表收入第{}张图片！".format(j), end="")
                all_md5.append(md5_file(file))


if __name__ == '__main__':
    files = file_count(filedir)
    # print(files)
    oldf = len(files)
    print('去重前有', oldf, '个文件\n请稍等正在删除重复文件...',flush=True)
    delfile()
    files=[]
    ndf = len(file_count(filedir))
    print('\n\n去重后剩', ndf, '个文件')
    print('\n\n一共删除了', oldf - ndf, '个文件\n\n')
